/*37218898_306040569 Lihi_Dery_Luba_Golosman*/
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
//import java.util.Arrays;
import java.util.Collections;




public class InputParser {
	private static final Integer NUM_OCCUR = 4;
	private String filename;
	private String topicsFileName;
	private ArrayList<Document> output = new ArrayList<Document>();
	private Document corpus = new Document();
	private ArrayList<String> topics = new ArrayList<String>();


	public void parse()
	{
		try{
			  // Open the file that is the first 
			  // command line parameter
			  FileInputStream fstream = new FileInputStream(filename);
			  // Get the object of DataInputStream
			  DataInputStream in = new DataInputStream(fstream);
			  BufferedReader br = new BufferedReader(new InputStreamReader(in));
			  String strLine;
			  //Read File Line By Line
			  ArrayList<String> corpusWords = new ArrayList<String>();
			  while ((strLine = br.readLine()) != null)   {

				  if (strLine.startsWith("<") && strLine.endsWith(">"))
				  { 
					  ArrayList<String> topics = new ArrayList<String>();
					  Collections.addAll(topics, strLine.split("\t"));
					  topics.remove(0);
					  topics.remove(0);
					  int last_index= topics.size()-1;
					  topics.set(last_index,topics.get(last_index).substring(0, topics.get(last_index).length()-1));
					  if ((strLine = br.readLine()) != null)
					  {
						  ArrayList<String> documentWords = new ArrayList<String>();
						  Collections.addAll(documentWords, strLine.split(" "));
						  Collections.addAll(corpusWords, strLine.split(" "));
						  Document document = new Document();
						  document.setWords(documentWords);
						  document.setTopics(topics);
						  output.add(document);
					  }
				  }
			  }
			  corpus.setWords(corpusWords);
			  //Close the input stream
			  in.close();
			  
			  // Open the file that is the first 
			  // command line parameter
			  fstream = new FileInputStream(topicsFileName);
			  // Get the object of DataInputStream
			  in = new DataInputStream(fstream);
			  br = new BufferedReader(new InputStreamReader(in));

			  //Read File Line By Line
			  while ((strLine = br.readLine()) != null)   {
				  topics.add(strLine);
			  }

			}catch (Exception e){//Catch exception if any
			    	System.err.println("Error: " + e.getMessage());
			  }
			filterWords();
	 }

	private void filterWords() {
		Set<String> wordsToFilter = new HashSet<String>();
		corpus.InitializeEventMap();
		HashMap<String,Integer> corpusEventMap = corpus.getEventMap();
		for (Entry<String,Integer> entry : corpusEventMap.entrySet()) {
			if (entry.getValue() < NUM_OCCUR)
			{
				wordsToFilter.add(entry.getKey());
			}
		}
		for (Document doc : output)
		{
			doc.getWords().removeAll(wordsToFilter);
		}
		corpus.getWords().removeAll(wordsToFilter);
		corpus.setEventMap(new HashMap<String, Integer>());
	}

	public String getFilename() {
		return filename;
	}

	public void setFilesName(String filename ,String topicsFileName) {
		this.filename = filename;
		this.topicsFileName = topicsFileName;
	}

	public ArrayList<Document> getDocuments() {
		return output;
	}

	public void setOutput(ArrayList<Document> output) {
		this.output = output;
	}

	public Document getCorpus() {

		return corpus;
	}
	public ArrayList<String> getTopics() {
		return topics;
	}
}
